#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018-04-12 14:08
# @Author : Woolei
# @File : Step1_getSpecialistSubject.py

"""
获取主页期刊的各大类别
https://www.tandfonline.com/
"""

import facade
from bs4 import BeautifulSoup
from xjlibrary.configread import MyConfigParser
from xjlibrary.our_file_dir.base_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")

# 处理数据库语句中的转义
dealSqlTextFunc = lambda text: text.replace("'", "''") if isinstance(text, str) else text


class DownSubject(object):

    def __init__(self):
        self.HOME_URL = 'https://www.tandfonline.com/'
        self.HEADERS = {
            'referer': 'http://taylorandfrancis.com/journals/',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
        }
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(configfile, "db", logger=self.logger)
        self.ini_dicts = MyConfigParser().set_config_path(
            configfile).get_configparser().set_keep_keys_case().read_config_string(
            'utf-8').conf_to_dict()

    def downsubject(self):
        BoolResult, errString, r = facade.BaseRequest(self.HOME_URL,
                                                      headers=self.HEADERS,
                                                      verify=False,
                                                      timeout=(30, 60))
        if BoolResult:
            for catid, text, extra_url in self.para_home(r.text):
                if catid and text and extra_url:
                    if self.ini_dicts["other"]["is_all"] == "true":
                        sql = "replace into subject(catid, subject, extra_url) VALUES ('%s','%s','%s')" % (
                            catid, dealSqlTextFunc(text), dealSqlTextFunc(extra_url))
                        self.mysqlutils.ExeSqlToDB(sql)
                        self.logger.info('Insert subject: %s, Extra_url: %s' % (text, extra_url))
                    else:
                        sql = "insert ignore into subject(catid, subject, extra_url) VALUES ('%s','%s','%s')" % (
                            catid, dealSqlTextFunc(text), dealSqlTextFunc(extra_url))
                        self.mysqlutils.ExeSqlToDB(sql)
                        self.logger.info('Insert subject: %s, Extra_url: %s' % (text, extra_url))
                        sql = "update subject set stat=0,failcount=0"
                        self.mysqlutils.ExeSqlToDB(sql)

    def para_home(self, html):
        bsObj = BeautifulSoup(html, 'lxml')
        if bsObj:
            content_1 = bsObj.find_all('div', {'class': 'unit'})
            for con in content_1:
                for c in con.find_all('a'):
                    extra_url, text = c['href'], c.get_text()
                    catid = extra_url.split('/')[-1].split('?')[0]
                    yield catid, text, extra_url


def main():
    down = DownSubject()
    down.downsubject()


if __name__ == '__main__':
    main()
